{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第五步：调整正则化参数：reg_alpha 和reg_lambda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "\n",
    "import xgboost as xgb\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from matplotlib import pyplot\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "import seaborn as sns\n",
    "\n",
    "from numpy import nan as NaN\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Filled_Form_N</th>\n",
       "      <th>Filled_Form_Y</th>\n",
       "      <th>Device_Type_Mobile</th>\n",
       "      <th>Device_Type_Web-browser</th>\n",
       "      <th>Mobile_Verified_N</th>\n",
       "      <th>Mobile_Verified_Y</th>\n",
       "      <th>Source_S122</th>\n",
       "      <th>Source_S123</th>\n",
       "      <th>Source_S124</th>\n",
       "      <th>Source_S127</th>\n",
       "      <th>...</th>\n",
       "      <th>Loan_Amount_Submitted</th>\n",
       "      <th>Loan_Tenure_Submitted</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Processing_Fee</th>\n",
       "      <th>Disbursed</th>\n",
       "      <th>DOB_month</th>\n",
       "      <th>DOB_year</th>\n",
       "      <th>age</th>\n",
       "      <th>Lead_Creation_Date_month</th>\n",
       "      <th>Lead_Creation_Date_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>620000.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>13.99</td>\n",
       "      <td>3100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8</td>\n",
       "      <td>1987</td>\n",
       "      <td>32</td>\n",
       "      <td>7</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>260000.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>33.00</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1993</td>\n",
       "      <td>26</td>\n",
       "      <td>7</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>100000.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>28.50</td>\n",
       "      <td>6600.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1978</td>\n",
       "      <td>41</td>\n",
       "      <td>7</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>200000.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>28.50</td>\n",
       "      <td>5000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6</td>\n",
       "      <td>1985</td>\n",
       "      <td>34</td>\n",
       "      <td>7</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>300000.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>16.25</td>\n",
       "      <td>7000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "      <td>1975</td>\n",
       "      <td>44</td>\n",
       "      <td>7</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 43 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Filled_Form_N  Filled_Form_Y  Device_Type_Mobile  Device_Type_Web-browser  \\\n",
       "0              0              1                   1                        0   \n",
       "1              1              0                   1                        0   \n",
       "2              1              0                   0                        1   \n",
       "3              1              0                   0                        1   \n",
       "4              1              0                   0                        1   \n",
       "\n",
       "   Mobile_Verified_N  Mobile_Verified_Y  Source_S122  Source_S123  \\\n",
       "0                  0                  1            1            0   \n",
       "1                  0                  1            1            0   \n",
       "2                  0                  1            1            0   \n",
       "3                  1                  0            1            0   \n",
       "4                  0                  1            1            0   \n",
       "\n",
       "   Source_S124  Source_S127           ...             Loan_Amount_Submitted  \\\n",
       "0            0            0           ...                          620000.0   \n",
       "1            0            0           ...                          260000.0   \n",
       "2            0            0           ...                          100000.0   \n",
       "3            0            0           ...                          200000.0   \n",
       "4            0            0           ...                          300000.0   \n",
       "\n",
       "   Loan_Tenure_Submitted  Interest_Rate  Processing_Fee  Disbursed  DOB_month  \\\n",
       "0                    4.0          13.99          3100.0        0.0          8   \n",
       "1                    4.0          33.00          2000.0        0.0          2   \n",
       "2                    5.0          28.50          6600.0        0.0          2   \n",
       "3                    3.0          28.50          5000.0        0.0          6   \n",
       "4                    5.0          16.25          7000.0        0.0          4   \n",
       "\n",
       "   DOB_year  age  Lead_Creation_Date_month  Lead_Creation_Date_year  \n",
       "0      1987   32                         7                     2015  \n",
       "1      1993   26                         7                     2015  \n",
       "2      1978   41                         7                     2015  \n",
       "3      1985   34                         7                     2015  \n",
       "4      1975   44                         7                     2015  \n",
       "\n",
       "[5 rows x 43 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('FE_X_train.csv')\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop ids and get labels\n",
    "y_train = train['Disbursed']\n",
    "\n",
    "train = train.drop([\"Disbursed\"], axis=1)\n",
    "X_train = np.array(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [1.5, 2], 'reg_lambda': [0.5, 1, 2]}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#reg_alpha = [1e-3, 1e-2, 0.05, 0.1]    #default = 0\n",
    "#reg_lambda = [1e-3, 1e-2, 0.05, 0.1]   #default = 1\n",
    "\n",
    "reg_alpha = [ 1.5, 2]    #default = 0, 测试0.1,1，1.5，2\n",
    "reg_lambda = [0.5, 1, 2]      #default = 1，测试0.1， 0.5， 1，2\n",
    "\n",
    "param_test5_1 = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)\n",
    "param_test5_1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'reg_alpha': [1.5, 2], 'reg_lambda': [0.5, 1, 2]},\n",
       " {'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       " -0.3898998721141688)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)\n",
    "\n",
    "xgb5_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=6,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=2,\n",
    "        min_child_weight=0.001,\n",
    "        gamma=0,\n",
    "        subsample=0.7,\n",
    "        colsample_bytree=0.9, #调整后\n",
    "        colsample_bylevel = 0.7,#调整后\n",
    "        objective= 'binary:logistic',\n",
    "#         num_class = 9,\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch5_1 = GridSearchCV(xgb5_1, param_grid = param_test5_1, scoring='neg_log_loss',n_jobs=2, cv=kfold)\n",
    "gsearch5_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch5_1.param_grid, gsearch5_1.best_params_, gsearch5_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([0.04114137, 0.03835516, 0.03798242, 0.0385848 , 0.03845539,\n",
       "        0.0376687 ]),\n",
       " 'std_fit_time': array([0.00208366, 0.00103699, 0.00116453, 0.00085818, 0.00102194,\n",
       "        0.00028634]),\n",
       " 'mean_score_time': array([0.00265698, 0.00241513, 0.00255437, 0.00231652, 0.00238018,\n",
       "        0.00240278]),\n",
       " 'std_score_time': array([2.76995579e-04, 7.72119075e-05, 1.23751269e-04, 1.12949870e-04,\n",
       "        1.40328841e-04, 1.48402572e-04]),\n",
       " 'param_reg_alpha': masked_array(data=[1.5, 1.5, 1.5, 2, 2, 2],\n",
       "              mask=[False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_reg_lambda': masked_array(data=[0.5, 1, 2, 0.5, 1, 2],\n",
       "              mask=[False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " 'split0_test_score': array([-0.38852605, -0.38922763, -0.38969404, -0.38917168, -0.38940516,\n",
       "        -0.38986998]),\n",
       " 'split1_test_score': array([-0.38950488, -0.38980704, -0.39033529, -0.38981471, -0.39005175,\n",
       "        -0.39109102]),\n",
       " 'split2_test_score': array([-0.38987885, -0.38917294, -0.38999107, -0.38944668, -0.38969711,\n",
       "        -0.39019575]),\n",
       " 'split3_test_score': array([-0.39093044, -0.39118128, -0.39168059, -0.39112683, -0.39137731,\n",
       "        -0.39187593]),\n",
       " 'split4_test_score': array([-0.39066067, -0.39111851, -0.39191234, -0.39110705, -0.39157959,\n",
       "        -0.39241967]),\n",
       " 'mean_test_score': array([-0.38989987, -0.39010107, -0.39072219, -0.390133  , -0.39042172,\n",
       "        -0.39108993]),\n",
       " 'std_test_score': array([0.00085849, 0.00088457, 0.00090279, 0.00082854, 0.0008886 ,\n",
       "        0.00096666]),\n",
       " 'rank_test_score': array([1, 2, 5, 3, 4, 6], dtype=int32),\n",
       " 'split0_train_score': array([-0.3894055 , -0.38984489, -0.39032528, -0.38979185, -0.39003219,\n",
       "        -0.3905105 ]),\n",
       " 'split1_train_score': array([-0.38949285, -0.38974397, -0.39027576, -0.38988656, -0.39012585,\n",
       "        -0.39083715]),\n",
       " 'split2_train_score': array([-0.38936857, -0.38930474, -0.38998569, -0.38952027, -0.38976921,\n",
       "        -0.39026492]),\n",
       " 'split3_train_score': array([-0.38859183, -0.38883712, -0.38932564, -0.38877764, -0.38902267,\n",
       "        -0.38951069]),\n",
       " 'split4_train_score': array([-0.38947934, -0.38979471, -0.39068709, -0.38988395, -0.3903683 ,\n",
       "        -0.39157431]),\n",
       " 'mean_train_score': array([-0.38926762, -0.38950509, -0.39011989, -0.38957205, -0.38986365,\n",
       "        -0.39053951]),\n",
       " 'std_train_score': array([0.00034101, 0.00038542, 0.00045542, 0.00041911, 0.00046217,\n",
       "        0.00067737])}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch5_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # summarize results\n",
    "# print(\"Best: %f using %s\" % (gsearch5_1.best_score_, gsearch5_1.best_params_))\n",
    "# test_means = gsearch5_1.cv_results_[ 'mean_test_score' ]\n",
    "# test_stds = gsearch5_1.cv_results_[ 'std_test_score' ]\n",
    "# train_means = gsearch5_1.cv_results_[ 'mean_train_score' ]\n",
    "# train_stds = gsearch5_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "# pd.DataFrame(gsearch5_1.cv_results_).to_csv('my_preds_reg_alpha_reg_lambda_1.csv')\n",
    "\n",
    "# # plot results\n",
    "# test_scores = np.array(test_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "# train_scores = np.array(train_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "\n",
    "# #log_reg_alpha = [0,0,0,0]\n",
    "# #for index in range(len(reg_alpha)):\n",
    "# #   log_reg_alpha[index] = math.log10(reg_alpha[index])\n",
    "    \n",
    "# for i, value in enumerate(reg_alpha):\n",
    "#     pyplot.plot(reg_lambda, -test_scores[i], label= 'reg_alpha:'   + str(value))\n",
    "# #for i, value in enumerate(min_child_weight):\n",
    "# #    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "# pyplot.legend()\n",
    "# pyplot.xlabel( 'reg_alpha' )                                                                                                      \n",
    "# pyplot.ylabel( '-Log Loss' )\n",
    "# pyplot.savefig( 'reg_alpha_vs_reg_lambda1.png' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.3898998721141688"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch5_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': 1.5, 'reg_lambda': 0.5}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch5_1.best_params_"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
